package org.nerve.study.sprider.crawler4j;

import com.j256.ormlite.dao.Dao;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.nerve.study.sprider.common.DBUtil;
import org.nerve.study.sprider.common.StringUtil;
import org.nerve.study.sprider.domain.WebPage;

import java.net.SocketTimeoutException;
import java.util.Date;
import java.util.Set;
import java.util.regex.Pattern;

/**
 * org.nerve.study.sprider.crawler4j
 * Created by zengxm on 2016/3/17 0017.
 */
public class OnionCrawler extends CommonCrawler {
	@Override
	public boolean shouldVisit(Page referringPage, WebURL url) {
		String href = url.getURL().toLowerCase();
		if (FILTERS.matcher(href).matches()) {
			return false;
		}
		boolean should=StringUtil.extractDomain(href).contains(".onion");
		if(should){
			logger.info("可以采集（符合条件的地址）:"+href);

			crawlC++;
		}
		return should;
	}

	private long timeoutC=0;
	private long crawlC=0;
	private long successC=0;

	@Override
	protected void onVisited(Page page) {
		successC++;
		System.out.println("success crawl "+page.getWebURL());
		printCount();
	}

	@Override
	protected void onUnhandledException(WebURL webUrl, Throwable e) {
		logger.error("-------------------error:"+e.getMessage());
		if(e instanceof SocketTimeoutException){
			timeoutC++;
			printCount();
		}
	}

	private void printCount(){
		logger.info("current count= need crawl:"+crawlC+", success:"+successC+",timeout:"+timeoutC);
	}
}
